# Scrapy settings for gaodeha_crawl project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'gaodeha_crawl'

SPIDER_MODULES = ['gaodeha_crawl.spiders']
NEWSPIDER_MODULE = 'gaodeha_crawl.spiders'

DOWNLOADER_MIDDLEWARES = {
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware' : None,
'gaodeha_crawl.common.rotate_useragent.RotateUserAgentMiddleware' : 400,
'gaodeha_crawl.common.filter_request.FilterRequestMiddleware' : 100
}


ITEM_PIPELINES = {
'gaodeha_crawl.pipelines.ImageDbPipeline': 100,
'gaodeha_crawl.pipelines.PostDbPipeline': 300,
}

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36'

DOWNLOAD_DELAY = 10

DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue'
